# -*- coding: utf-8 -*-
import model
import re
import urllib2
import filestore
import urlparse

skip_re = re.compile("<h4>Incoming search terms:</h4><ul>.+?</ul>")

a_re = re.compile("(<a.+?href\s*=\s*[\"'].+?www\.21andy\.com.+?[\"']>(.+?)</a>)")

tb = model.mongo.topic
count = tb.find().count()
cursor = tb.find()

def get_and_store(img_url):
    try:
        request = urllib2.Request(img_url)
        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11')
        response = urllib2.urlopen(request)
        content = response.read()
        info = response.info()
    except:
        return None
    if(response.getcode() != 200 or info.subtype not in ["jpg","JPG",'jpeg','JPEG',"png","PNG","gif","GIF"]):
        return False
    f = filestore.FileStore()
    objid = f.get_objid_by_name(img_url)
    if(objid):
        return objid
    return f.put(content,img_url)

for i in cursor:
    content = i["content"]
    objid = i["_id"]
    tmp = skip_re.findall(content)
    t_c = None
    if(len(tmp) > 0):
        t_c = content.replace(tmp[0],"")
        content = t_c
    tmp = a_re.findall(content)
    if(len(tmp) > 0):
        for n in tmp:
            t_c = content.replace(n[0],n[1])
            content = t_c
    #增加图片处理
    img_re = re.compile("<img[^>]+?src=[\"'](.+?)[\"']",re.IGNORECASE)
    scanner = img_re.scanner(content)
    p = scanner.search()
    new_content = content;
    while(p):
        regs = p.regs
        p = scanner.search()
        if(len(regs) == 0):
            continue
        img = content[regs[0][0]:regs[0][1]]
        img_url = content[regs[1][0]:regs[1][1]]
        lup = urlparse.urlparse(img_url)
        if(lup.scheme == 'mailto' or lup.scheme == 'javascript' or (lup.netloc =='' and  lup.path == '') ):
            continue
        url_tmp = ""
        url_tmp = img_url
        print(url_tmp)
        fileid = get_and_store(url_tmp)
        if(fileid):
            new_img = img.replace(url_tmp,"/img/" + fileid)
            new_content = new_content.replace(img,new_img)
    tb.update({"_id":objid},{"$set":{"content":new_content}})
